# !/usr/bin/env python
# coding=utf-8

from vavava.workqueue import Work
import spider_cl,dbhelper_cl
import sys


class TaskWorkCL(Work):
    def __init__( self, period, group, begin_time=None, end_time=None, log=None,**kargs ):
        Work.__init__( self,period=period, group=group, begin_time=begin_time, end_time=end_time, log=log, name="" )
        self.__deepth = 2
        self.__dbpath = r"C:\fetch_cl\db\fetch_cl.db3"
        self.proxy_   = None
        if kargs is not None and "proxy" in kargs:
            self.proxy_ = kargs["proxy"]
        self.redo_counter_ = sys.maxsize

    def do(self,worker):
        if not worker:
            raise Exception( "[%s] Unknown parent,cannot add work for main task. " % self._name )
        if worker._db_conn is None:
            raise Exception( "[%s] DBObj is Null." % self._name )
        try:
            url_list = open(r"c:\fetch_cl\url_list.urls")
            seed_urls = url_list.readlines()
            for category_index in range(len(seed_urls)):
                seed_url = seed_urls[category_index]
            for i in range(self.__deepth):
                url = seed_url%(i+1)
                self.log.debug("add item:%s"%url)
                w = spider_cl.FecthWork (url=url,category=category_index, dbpath=self.__dbpath,log=self.log)
                worker.add_work_to_parent(w)
        except Exception as e:
            self.log.error( "[%s] exception in _main_routine", self._name )
            self.log.exception(e)

